import random
import sys
import time
from datetime import datetime

from spidertools.utils.time_utils import get_current_datetime, get_current_date
from commonresources.inner_utils import mongdb_utils
# from crawlab import save_item
import os

from pymongo.errors import DuplicateKeyError

from commonresources.inner_utils.dd_warning_msg import ddmsgsend


class ProjectHtmlPipeline(object):
    def __init__(self):
        self.myclient, self.mydb = mongdb_utils.get_mongo_connect(mongdb_utils.default_mongo_url)
        # self.myclient, self.mydb = mongdb_utils.get_mongo_connect("mongodb://192.168.2.181:27017")
        self.project_env = os.environ.get("PROJECT_ENV")

        if self.project_env != None and self.project_env == 'test':
            self.mycol = self.mydb['project_html_test']
        else:
            self.mycol = self.mydb['project_html']
        self.mycol.create_index([("origin_url", 1)], unique=True)  # 设置唯一
        self.total_count = 0

    def start_spider(self, spider):
        pass

    def close_spider(self, spider):
        # 这是一个结果，需要为 dict 类型
        current_date = get_current_date()
        result = {current_date: self.total_count}
        # 调用保存结果方法
        # save_item(result)
        if spider.name == 'ShanDongShengGongGongZiYuanJiaoYiWang':
            spider.driver.quit()  # 结束时退出 driver
            print("退出driver")
        # if not self.total_count and not sys.platform.startswith('win') and datetime.now().weekday() < 5:
        #     msg = f"1.众能联合_工程信息知识库项目 {get_current_datetime()}\n" \
        #           f"2.{spider.name} {spider.name_zh}\n" \
        #           f"3.采集数据为0，请予以关注"
        #     ddmsgsend(msg)
        # else:
        #     finish_msg = f"1.众能联合_工程信息知识库项目 {get_current_datetime()}\n" \
        #                  f"2.【{spider.name_zh}】  {spider.name}\n" \
        #                  f"3.本次采集数据量为【 {self.total_count} 】，已完成采集，请知悉"
        #     ddmsgsend(finish_msg)
        self.myclient.close()

    def process_item(self, item, spider):
        insert_item = dict(item)
        if hasattr(spider, "convert_dict") and spider.convert_dict:
            new_insert_item = {}
            for key, value in insert_item.items():
                if key in spider.convert_dict:
                    new_insert_item[spider.convert_dict[key]] = value
                else:
                    new_insert_item[key] = value
            insert_item = new_insert_item
        if not spider.full_dose:
            # self.mycol.insert_one(insert_item)
            search_dict = {"origin_url": insert_item['origin_url']}

            search_result = self.mycol.find(search_dict).count()
            if search_result:
                print("update!!!!!!!")
                insert_item['update_time'] = get_current_datetime()
                update_value = {"$set": insert_item}
                self.mycol.update_one(search_dict, update_value)
            else:
                print("insert!!!!!!!")
                insert_item['create_time'] = get_current_datetime()
                insert_item['update_time'] = get_current_datetime()
                self.mycol.insert_one(insert_item)
        else:
            try:
                insert_item['create_time'] = get_current_datetime()
                insert_item['update_time'] = get_current_datetime()
                self.mycol.insert_one(insert_item)
                print("insert1111!!!!!...")
            except DuplicateKeyError:
                insert_item['update_time'] = get_current_datetime()
                update_value = {"$set": insert_item}
                del insert_item['_id']
                self.mycol.update_one({"origin_url": insert_item['origin_url']}, update_value)
                print("update1111!!!!!...")

        # save_item({"origin_url": insert_item["origin_url"]})
        self.total_count += 1
        # print("=============", type(item), "\t", self.total_count)
        return item


class HangYeCompanyCountPipeline(object):
    def __init__(self):
        default_mongo_url_string = "mongodb://192.168.2.181:27017"
        # self.myclient, self.mydb = mongdb_utils.get_mongo_connect(mongdb_utils.default_mongo_url)
        self.myclient, self.mydb = mongdb_utils.get_mongo_connect(default_mongo_url_string)
        self.project_env = os.environ.get("PROJECT_ENV")

        if self.project_env != None and self.project_env == 'test':
            self.mycol = self.mydb['demand_3']
        else:
            self.mycol = self.mydb['demand_3']

    def start_spider(self, spider):
        pass

    def close_spider(self, spider):
        self.myclient.close()

    def process_item(self, item, spider):
        insert_item = dict(item)
        if hasattr(spider, "convert_dict") and spider.convert_dict:
            new_insert_item = {}
            for key, value in insert_item.items():
                if key in spider.convert_dict:
                    new_insert_item[spider.convert_dict[key]] = value
                else:
                    new_insert_item[key] = value
            insert_item = new_insert_item
        search_key = {"所在城市": insert_item['所在城市']}
        # search_key = {"city": insert_item['city']}

        # dict(dict(search_key).keys())

        search_results = self.mycol.find(search_key)
        count = self.mycol.find(search_key).count()
        if count:
            for search_result in search_results:
                item = dict(search_result)
                if item:
                    tem = {}
                    for it in insert_item:
                        if it not in item:
                            tem[it] = insert_item[it]
                    try:
                        self.mycol.update_one(search_key, {"$set": tem})
                    except Exception as e:
                        print("error_log", search_result)
        else:
            self.mycol.insert_one(insert_item)
        return item
